﻿//////////////////////////////////////////////////////////////////
//
// Bruno\Code.cs
//
// Copyright (c) 2012 Dan Pike.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation files
// (the "Software"), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Utilities;
using System.Drawing;
using System.Windows.Forms;
using System.IO;

using HtmlDocument = HtmlAgilityPack.HtmlDocument;
using System.Globalization;
using System.Threading;

namespace Bruno
{
   public class Code
   {
      private static Logger logger_;
      private static readonly string[] INIT_SUBITEMS;
      public string Name;
      public string CurrentFilename;
      public double MinAge;
      private Config.Code settings_;
      private ListViewItem item_;
      private string filenameFormat_;
      private string intraDayFormat_;
      private static readonly CultureInfo enGB;

      public enum PageState
      {
         None,          // Initial state; not a valid state for pages
         NeedCheck,     // The state of this page is not known
         WantRequest,   // Want to request this page from the server
         Requesting,    // We have requested this page and we are waiting for the server to supply it
         ParseFailed,   // We seem to have data for this page but it failed to parse. Request again.
         Complete,      // We have received a full image for this page and it's not the last in the sequence
         Partial,       // We have received a full image, it's got no "Next" link but it's not yet old enough to be confident that there is no more
         Final,         // This is the last page for this code. The final entry is old enough that we can assume no more will come available
         Empty,         // This page does not exist; there is no data for it and never will be!
      }

      /// <summary>
      /// Store page states by code date. The index in the List<> corresponds to the page index
      /// There may be "holes" in the List<> that represent pages for which we have no image, yet
      /// </summary>
      private Dictionary<DateTime, List<PageState>> pageStates_;

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      static Code()
      {
         logger_ = Program.Logger;
         enGB = new CultureInfo("en-GB");

         INIT_SUBITEMS = new string[((Main)Program.MainForm).CodeListView.Columns.Count - 1];
         for (int index = 0; index < INIT_SUBITEMS.Length; ++index)
         {
            INIT_SUBITEMS[index] = string.Empty;
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      public Code()
      {
         MinAge = 0.0;  // On the first run, it's an instant check. The subsequent
                        // runs wait for an hour
         pageStates_ = new Dictionary<DateTime, List<PageState>>();
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      public Config.Code Settings
      {
         get { return settings_; }
         set
         {
            settings_ = value;
            filenameFormat_ = Path.Combine(Program.Settings.Folder,
               Settings.Folder, "{0:yyyyMMdd}.{1}.html");
            intraDayFormat_ = Path.Combine(Program.Settings.Folder,
               Settings.Folder, "{0:yyyyMMdd}.html");
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      public ListViewItem Item
      {
         get { return item_; }
         set
         {
            item_ = value;
            item_.Tag = this;
            item_.UseItemStyleForSubItems = false;
            item_.SubItems.AddRange(INIT_SUBITEMS);
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// Helper function for debug messages
      /// </summary>
      /// <param name="codeDate"></param>
      /// <param name="pageIndex"></param>
      /// <returns></returns>
      public string DebugName(DateTime codeDate, int pageIndex)
      {
         return string.Format("code<{0}, {1:dd/MM/yyyy}, {2}>", Name, codeDate, pageIndex);
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// Helper function for debug messages
      /// </summary>
      /// <param name="codeDate"></param>
      /// <returns></returns>
      public string DebugName(DateTime codeDate)
      {
         return string.Format("code<{0}, {1:dd/MM/yyyy}>", Name, codeDate);
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="codeDate"></param>
      public void SaveToArchive(DateTime codeDate)
      {
         //¬¬¬ SaveToArchive(): not yet implemented
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="codeDate"></param>
      /// <param name="pageIndex"></param>
      /// <returns></returns>
      public string GetFilename(DateTime codeDate, int pageIndex)
      {
         return string.Format(filenameFormat_, codeDate, pageIndex);
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="codeDate"></param>
      /// <returns></returns>
      public string GetIntradayFilename(DateTime codeDate)
      {
         return string.Format(intraDayFormat_, codeDate);
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="timeNow"></param>
      /// <param name="codeDate"></param>
      /// <param name="stopCheck"></param>
      /// <returns></returns>
      public bool CheckRequest(TimeSpan timeNow, DateTime codeDate, EventWaitHandle stopCheck)
      {
         bool changedSomething = false;
         string previousFilename, thisFilename = null;
         bool previousFileExists, thisFileExists = false;
         TimeSpan? thisFileWriteTime = null;
         PageState previousPageState, thisPageState = PageState.None;

         bool done = false;
         int pageIndex = 0;
         while (!done && !stopCheck.WaitOne(0))
         {
            previousFilename = thisFilename;
            thisFilename = GetFilename(codeDate, pageIndex);

            previousFileExists = thisFileExists;
            thisFileExists = File.Exists(thisFilename);

            thisFileWriteTime = thisFileExists ? (TimeSpan?)File.GetLastWriteTime(thisFilename).TimeOfDay : null;

            previousPageState = thisPageState;
            thisPageState = GetPageState(codeDate, pageIndex);
            switch (thisPageState)
            {
               case PageState.None:
                  throw new InvalidOperationException("unexpected PageState.None");

               case PageState.NeedCheck:
                  // We know nothing about this file, check it out
                  if (thisFileExists)
                  {
                     PageState thisFileState = GetFileState(thisFileWriteTime, thisFilename);
                     //logger_.DebugFormat("file '{0}' has state {1}", thisFilename, thisFileState);
                     thisPageState = SetPageState(codeDate, pageIndex, thisFileState);
                     changedSomething = true;

                     switch (thisFileState)
                     {
                        case PageState.ParseFailed:
                           // We should request this page
                           logger_.DebugFormat("we will request unknown/bad file '{0}'", thisFilename);
                           thisPageState = SetPageState(codeDate, pageIndex, PageState.WantRequest);
                           break;

                        case PageState.Final:
                           // Reached the end of the sequence
                           done = true;
                           break;

                        case PageState.Empty:
                           // The file is locked, skip it and try again later
                           thisPageState = SetPageState(codeDate, pageIndex, PageState.NeedCheck);
                           break;

                        case PageState.Partial:
                        case PageState.Complete:
                           // Nothing to do here
                           break;

                        default:
                           logger_.ErrorFormat("unexpected state '{0}' returned by GetFileState({1})",
                              thisFileState, thisFilename);
                           break;
                     }
                  }
                  else if (previousFileExists)
                  {
                     switch (previousPageState)
                     {
                        case PageState.Complete:
                           // We finished the previous file, so we request this new one
                           logger_.DebugFormat("we will request new page '{0}'", thisFilename);
                           thisPageState = SetPageState(codeDate, pageIndex, PageState.WantRequest);
                           changedSomething = true;
                           break;

                        case PageState.Empty:
                        case PageState.WantRequest:
                        case PageState.Final:
                        case PageState.Partial:
                           // Nothing to do here
                           break;

                        default:
                           // Need to decide what to do for the other states
                           logger_.ErrorFormat("unexpected state '{0}' for previous page when new file '{1}' does not exist",
                              previousPageState, thisFilename);
                           break;
                     }
                  }
                  else if (0 == pageIndex)
                  {
                     // No files at all?
                     CreateInitialSnapshot(timeNow, codeDate);
                     changedSomething = true;
                     done = true;
                  }
                  else
                  {
                     // We have reached the end of the files
                     done = true;
                  }
                  break;

               case PageState.WantRequest:
               case PageState.Requesting:
               case PageState.Complete:
               case PageState.Partial:
               case PageState.Final:
               case PageState.Empty:
                  // We've already analysed this file. Nothing more to do
                  break;

               default:
                  break;
            }
            ++pageIndex;
         }
         return changedSomething;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="codeDate"></param>
      private void CreateInitialSnapshot(TimeSpan timeNow, DateTime codeDate)
      {
         int pageCount = 0;
         double hourNow = timeNow.TotalHours;
         if (8.0 > hourNow)
         {
            // Put the first page on the pending list ready for when the markets open
            pageCount = 1;
         }
         else
         {
            if (17.0 < hourNow)
            {
               // Treat this as if the markets have just closed
               hourNow = 17.0;
            }
            // What's the average number of pages-per-hour that we take per day for this share (8am-4.30pm)?
            double averageCount = (0 < pageStates_.Count) ? pageStates_.Values.Average(pageStates => pageStates.Count) / 8.5 : 2.0f;

            // And how far through the day are we?
            double marketOpenTime = hourNow - 8.0;

            // So, how many pages should we try for?
            pageCount = Math.Max(1, (int)((averageCount * marketOpenTime) + 0.5));
         }

         logger_.DebugFormat("could not find any files for code {0}, date {1:dd/MM/yyyy} will request {2} pages",
            Name, codeDate, pageCount);
         for (int index = 0; index < pageCount; ++index)
         {
            SetPageState(codeDate, index, PageState.WantRequest);
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="writeTime"></param>
      /// <param name="filename"></param>
      /// <returns>
      /// One of:
      ///   PageState.ParseFailed, if the file doesn't look like a valid HTML file
      ///   PageState.Complete, if it's OK but there are more pages
      ///   PageState.Final, if it's the last page for this codeDate
      ///   PageState.Empty, if could not open the file (e.g. still writing to it!)
      ///   PageState.Partial, if we're still building the page
      /// </returns>
      private PageState GetFileState(TimeSpan? writeTime, string filename)
      {
         HtmlDocument doc = new HtmlDocument();
         try
         {
            doc.Load(filename);
         }
         catch (System.Exception ex)
         {
            // Try again later
            logger_.Exception(ex);
            return PageState.Empty;
         }

         // Does the page even parse?
         if ((0 < doc.ParseErrors.Count()) || (null == doc.DocumentNode))
         {
            return PageState.ParseFailed;
         }

         // Does it have a next link?
         if (null != GetNextUrl(doc))
         {
            // Yes, then it's complete but it's not the past page
            return PageState.Complete;
         }

         // There is no next link, is the time of the file after the market close on that
         // day? Given 30 minutes allowance, so check the time after 5pm
         if (17 < writeTime.Value.TotalHours)
         {
            // It's complete and, because there's no next link, it must be the final page
            return PageState.Final;
         }

         // The page is still building, i.e. market is still active
         return PageState.Partial;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="doc"></param>
      /// <returns></returns>
      public static bool ParseEntryTime(HtmlNode node, out DateTime entryTime)
      {
         string dateString = HtmlEntity.DeEntitize(node.FirstChild.InnerText).Trim();
         return DateTime.TryParseExact(dateString,
            "dd/MM/yy HH:mm", enGB, DateTimeStyles.None, out entryTime);
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="doc"></param>
      /// <returns></returns>
      public static HtmlNodeCollection GetAllNodes(HtmlDocument doc)
      {
         return doc.DocumentNode.SelectNodes("//tr[@class=\"stockrow\" or @class=\"stockrowhighlight\"]");
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// Returns the url for the Next link in this page if it has one
      /// </summary>
      /// <param name="doc"></param>
      /// <returns></returns>
      public static string GetNextUrl(HtmlDocument doc)
      {
         HtmlNode htmlNode = null;
         if ((null != doc) && (null != (htmlNode = doc.DocumentNode)))
         {
            htmlNode = htmlNode.SelectSingleNode("//a[text()=\"Next\"]");
         }
         return (null != htmlNode) ? htmlNode.GetAttributeValue("href", null) : null;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="codeDate"></param>
      /// <returns></returns>
      private PageState SetPageState(DateTime codeDate, int pageIndex, PageState pageState)
      {
         // Record the filename for this position
         List<PageState> pageStates;
         if (!pageStates_.TryGetValue(codeDate, out pageStates))
         {
            pageStates_[codeDate] = pageStates = new List<PageState>();            
         }
         while (pageIndex >= pageStates.Count)
         {
            // Need to check intermediary pages
            pageStates.Add(PageState.NeedCheck);
         }
         pageStates[pageIndex] = pageState;
         return pageState;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="codeDate"></param>
      /// <param name="pageIndex"></param>
      /// <returns></returns>
      private PageState GetPageState(DateTime codeDate, int pageIndex)
      {
         // Record the filename for this position
         List<PageState> pageStates;
         if (0 > pageIndex)
         {
            // Invalid page index
            return PageState.None;
         }
         if (!pageStates_.TryGetValue(codeDate, out pageStates) ||
            (pageStates.Count <= pageIndex))
         {
            // No such code date, or unknown page index, so we need to check it
            return PageState.NeedCheck;
         }
         return pageStates[pageIndex];
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// Returns the number of files that we have for the given date
      /// </summary>
      /// <param name="codeDate"></param>
      public int GetNumberOfPages(DateTime codeDate)
      {
         List<PageState> pageStates;
         if (!pageStates_.TryGetValue(codeDate, out pageStates))
         {
            return 0;
         }
         return pageStates.Count(pageState =>
               (PageState.Complete == pageState) ||
               (PageState.Partial == pageState) ||
               (PageState.Final == pageState)
            );
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// Are there any pages pending request?
      /// </summary>
      /// <param name="codeDate"></param>
      public bool GetWantPage(DateTime codeDate)
      {
         List<PageState> pageStates;
         if (!pageStates_.TryGetValue(codeDate, out pageStates))
         {
            return false;
         }
         return pageStates.Any(pageState => PageState.WantRequest == pageState);
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// All of the pages that we want to request
      /// </summary>
      /// <param name="codeDate"></param>
      /// <returns></returns>
      public List<int> GetWantedPages(DateTime codeDate)
      {
         List<PageState> pageStates;
         List<int> result = new List<int>();
         if (pageStates_.TryGetValue(codeDate, out pageStates))
         {
            int pageCount = pageStates.Count;
            for (int index = 0; index < pageCount; ++index)
            {
               if (PageState.WantRequest == pageStates[index])
               {
                  result.Add(index);
               }
            }
         }
         return result;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="codeDate"></param>
      /// <param name="pageIndex"></param>
      public void SetRequestingPage(DateTime codeDate, int pageIndex)
      {
         SetPageState(codeDate, pageIndex, PageState.Requesting);
      }
   }
}
